package com.foreveross.proxyip.core.http.ipdoload.website;

import java.lang.reflect.Constructor;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.params.CoreProtocolPNames;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import com.foreveross.proxyip.core.BaseCode;
import com.foreveross.proxyip.core.http.ipdoload.AbsProxyIpOriginal;
import com.foreveross.proxyip.entity.ExtractLogEntity;
import com.foreveross.proxyip.entity.IpWebsiteEntity;
import com.foreveross.proxyip.entity.ProxyIpBean;
import com.foreveross.proxyip.infra.base.ddd.repository.IBaseRepository;
import com.foreveross.proxyip.util.ThreadUtils;

/**
 * 从代理ip提供的页面获取源ip
 * @author lyq
 *
 */
@Service
public class WebSiteDoload extends AbsProxyIpOriginal {
	
private static final Logger log =  Logger.getLogger(WebSiteDoload.class);
protected String splitSign = "#";



private static Integer EXTRACT_CONNECTION_TIMEOUT = null; 

private static Integer EXTRACT_SO_TIMEOUT = null;

protected String encoding = null;

@Autowired
public WebSiteDoload(@Qualifier("entityHibernateRepository")IBaseRepository  baseRepository){
	super.baseRepository = baseRepository;
}

/**
 * 从下载器下载ip
 */
@Transactional	
public void extract(Map<String,ProxyIpBean> proxyIpMap,IpWebsiteEntity website,Integer num) throws Exception {
	 super.extract(proxyIpMap,website,num);
}

//IpstoreRepository ipstoreRepository;
/**
 * 获取代理ip集合
 * @param num
 * @return
 * @throws Exception
 */
@Override
	public void loadProxyIpMap(Map <String,ProxyIpBean> proxyIpMap,IpWebsiteEntity website,ExtractLogEntity extractLog,Integer num) throws Exception {
		if(proxyIpMap == null){
			proxyIpMap = new HashMap<String, ProxyIpBean>();
		}
		
//		List<IpStoreTmpEntity> tempList = new ArrayList<IpStoreTmpEntity>();
		Object[] ips = httpGetIPS(website,num);
		if(ips==null){
			log.info("从网上下载 LOST!");
				return ;//为空时调用 addIpsToData 方法要对NULL进行处理
		}else{
			log.info("从网上下载IP数："+ips.length+" SUCCESS!");
			extractLog.setExtractNum(ips.length);
		}
		
 		Constructor constructor = ExplainIpsCall.class.getConstructor(List.class,IpWebsiteEntity.class,Map.class,IBaseRepository.class);
		ThreadUtils.openThreads(constructor, Arrays.asList(ips), 200, 60000L, website,proxyIpMap,baseRepository);
		
		log.info("actually,current, web load ip size : " + proxyIpMap.size());
		
//改用jdbc插入数据库
//		IpstoreRepository ipstoreRepository = InstanceFactory.getInstance(IpstoreRepository.class);
//		ipstoreRepository.saveIpstores(tempList);
//		return list;
	}

//private void exp(Object[] ips, IpWebsiteEntity website,
//		Map<String, ProxyIpBean> proxyIpMap) throws IllegalAccessException,
//		InvocationTargetException, InstantiationException,
//		NoSuchMethodException {
//	for (int j = 0; j < ips.length; j++) {
//		if(RegUtil.getIpPort(ips[j].toString())!=null){
//			ProxyIpBean bean=new ProxyIpBean();
//			
////				list.add(bean);
////				this.setIpStoreEntity(bean,  ips[j].toString().trim());
//			this.explainRow(bean, website, ips[j].toString().trim());
//			IpStoreEntity oldIpStore = this.getIpStoreEntityByIp(bean.getIp());
//			
//			bean.setSource(website.getName());
//			bean.setCreateTime(new Date());
//			bean.setLastTime(new Date());
//			bean.setState(0);///还没被使用
////				tempList.add(bean.toIpStoreTmpEntity());
//			if(oldIpStore==null){
//				proxyIpMap.put(bean.getIp(),bean);
////					this.creatNew(bean);
//				BufferInstance bufferinstance = new BufferInstance(bean.toIpStoreEntity(), Operations.CREATE);
//				BufferCacheManager.getInstance().excure(bufferinstance,"ipStore");
//			}else if(oldIpStore.getCreateTime().getTime()<System.currentTimeMillis()-60*30*1000){
//				//如果数据库已存在该ip且上次存在于程序的时间已经超过半个钟
//				IpStoreEntity newStore = (IpStoreEntity)BeanUtils.cloneBean(oldIpStore);//去缓存
//				newStore.setLastTime(new Date());
//				BufferInstance bufferinstance = new BufferInstance(newStore, Operations.UPDATE);
//				BufferCacheManager.getInstance().excure(bufferinstance,"ipStore");
//				proxyIpMap.put(bean.getIp(),bean);
//			}
//		}
//	}
//}

	
//	/**
//	 * 分析 
//	 * @param httpStr
//	 * @return
//	 */
//	public abstract void setIpStoreEntity(ProxyIpBean ipStoreEntity,String httpStr);
	
	/**
	 * 子类实现 下载IP
	 * @param i
	 * @return
	 * @throws Exception
	 */
//	public abstract Object[] httpGetIPS(IpWebsiteEntity website,Integer num) throws Exception;
	

	
//	/**
//	 * 子类实现返回Class名称
//	 * @return
//	 */
//	public abstract Class returnClass();
	
	protected HttpClient getHttpClient() throws NumberFormatException, Exception{
		if(EXTRACT_CONNECTION_TIMEOUT==null){
			EXTRACT_CONNECTION_TIMEOUT = Integer.parseInt( BaseCode.getValue("EXTRACT_CONNECTION_TIMEOUT"));
		}
		if(EXTRACT_SO_TIMEOUT==null){
			EXTRACT_SO_TIMEOUT = Integer.parseInt( BaseCode.getValue("EXTRACT_SO_TIMEOUT"));
		}
		
		DefaultHttpClient httpclient=null;
		// 设置 user agent 
		String USER_AGENT = "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 GTB6 (.NET CLR 3.5.30729)";
		HttpParams httpParams = new BasicHttpParams();
		HttpProtocolParams.setUserAgent(httpParams, USER_AGENT);
		httpParams.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,
				EXTRACT_CONNECTION_TIMEOUT);
		httpParams.setParameter(CoreConnectionPNames.SO_TIMEOUT, EXTRACT_SO_TIMEOUT);
		httpParams.setBooleanParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE, false);
		 httpclient = new DefaultHttpClient(httpParams);
         return httpclient;
	}
	
	
	
	protected  String  getHttpPage(HttpRequestBase http,String encoding) throws Exception{
	     HttpResponse response =getHttpClient().execute(http);   
	     HttpEntity entity = response.getEntity();
	     if(StringUtils.isBlank(encoding)){
	    	return EntityUtils.toString(entity);
	     }else{
	    	 return EntityUtils.toString(entity,encoding);
	     }
	}
	
	@SuppressWarnings("rawtypes")
	private void creatNew(ProxyIpBean proxyIpBean) throws Exception{
//		BufferCacheManager bufferManger = new BufferCacheManager(proxyIpBean.toIpStoreEntity());
//		bufferManger.excure(Operations.CREATE,"ipstore");
//		Object []params = {proxyIpBean.getIp()};
//		List list = this.baseRepository.queryResult("from IpStoreEntity t where t.ip=?", params);
//		if(list==null||list.size()==0){
			baseRepository.save(proxyIpBean.toIpStoreEntity());
//		}
	}
	
		
	@SuppressWarnings("rawtypes")
	public Object [] httpGetIPS(IpWebsiteEntity website,Integer num) throws Exception {
		return this.httpGetIPS(num, new HashSet(),website, 1);
	}
		
		/**
		 * 最多递归抽取invokeCount次，如果抽取不够ip返回
		 * @param num 抽取ip数
		 * @param set 抽取ip集合的保存集合
		 * @param invokeCount 最大抽取次数
		 * @return
		 * @throws Exception
		 */
		@SuppressWarnings({ "unchecked", "rawtypes" })
		private Object[] httpGetIPS(Integer num,Set set ,IpWebsiteEntity website,int invokeCount) throws Exception {
			String url = getSiteUrl(website,num-set.size());
			HttpGet httpGet=new HttpGet(url);
			String sHtml=getHttpPage(httpGet,website.getEncoding());
			String [] arr = sHtml.split("\n");
			List tempList = Arrays.asList(arr);
			set.addAll(tempList);
			--invokeCount;
			if(set.size()<num&&invokeCount>0){
				return httpGetIPS(num,set,website,1);
			}else{
				return set.toArray();
			}
		}
		
		
		private String getSiteUrl(IpWebsiteEntity website,Integer num){
			String url = website.getUrlRegex();
			if(StringUtils.isNotBlank(website.getOrderid())){
				url = StringUtils.replace(url,"{orderid}", website.getOrderid());
			}
			url = StringUtils.replace(url,"{num}", num+"");
			return url;
		}

}

